{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-5250476a73f0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     33\u001b[0m     \u001b[0mXi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mYi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbootstrapData\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mYt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mM\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# draw this member's random sample of data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m     \u001b[0mrforest\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtreeClassify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# and train the model on that draw\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m     \u001b[0mrforest\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mYi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmaxDepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m25\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnFeatures\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m     \u001b[0mYtHat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrforest\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# predict on training data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m     \u001b[0mYvHat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrforest\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXv\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# and validation data & save results\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, X, Y, *args, **kwargs)\u001b[0m\n\u001b[1;32m    282\u001b[0m         \"\"\"\n\u001b[1;32m    283\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 284\u001b[0;31m         \u001b[0mtreeBase\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mto1ofK\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    285\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    286\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, X, Y, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m     92\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m              \u001b[0;31m# start building at the root\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mL\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mL\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m]\u001b[0m                              \u001b[0;31m# store returned data into object\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    163\u001b[0m         \u001b[0;31m# recur left\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mL\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mR\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    163\u001b[0m         \u001b[0;31m# recur left\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mL\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    163\u001b[0m         \u001b[0;31m# recur left\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mL\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mR\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    163\u001b[0m         \u001b[0;31m# recur left\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mL\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    163\u001b[0m         \u001b[0;31m# recur left\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mL\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mR\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mR\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    163\u001b[0m         \u001b[0;31m# recur left\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mL\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mR\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mR\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mR\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    163\u001b[0m         \u001b[0;31m# recur left\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mL\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mR\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mR\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mR\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    163\u001b[0m         \u001b[0;31m# recur left\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mL\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    163\u001b[0m         \u001b[0;31m# recur left\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mL\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_left\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# recur right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mR\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmy_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__train_recursive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgo_right\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminParent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxDepth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminLeaf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnFeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36m__train_recursive\u001b[0;34m(self, X, Y, depth, minParent, maxDepth, minLeaf, nFeatures)\u001b[0m\n\u001b[1;32m    144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    145\u001b[0m             \u001b[0;31m# find min weighted variance among split points\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 146\u001b[0;31m             \u001b[0mval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0midx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata_impurity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtsorted\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcan_split\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    147\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    148\u001b[0m             \u001b[0;31m# save best feature and split point found so far\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/Documents/UCI/UCI homework/2017 winter/CS178/Kaggle/Kaggle-code/mltools/dtree.pyc\u001b[0m in \u001b[0;36mentropy\u001b[0;34m(tsorted, can_split)\u001b[0m\n\u001b[1;32m    312\u001b[0m         \u001b[0meps\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspacing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    313\u001b[0m         \u001b[0;31m#y_left = np.cumsum(to1ofK(tsorted, self.classes), axis=0).astype(float)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 314\u001b[0;31m         \u001b[0my_left\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcumsum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtsorted\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    315\u001b[0m         \u001b[0my_right\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my_left\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0my_left\u001b[0m         \u001b[0;31m# construct p(class) for each possible split\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    316\u001b[0m         \u001b[0mwts_left\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1.0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m     \u001b[0;31m# by counting & then normalizing by left/right sizes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/LeoLu/anaconda/lib/python2.7/site-packages/numpy/core/fromnumeric.pyc\u001b[0m in \u001b[0;36mcumsum\u001b[0;34m(a, axis, dtype, out)\u001b[0m\n\u001b[1;32m   2113\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2114\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0m_wrapit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'cumsum'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2115\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mcumsum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2116\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2117\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#Random Forest\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import mltools as ml\n",
    "import mltools.dtree as dt;\n",
    "import sys\n",
    "sys.path.append('.')\n",
    "reload(dt);\n",
    "X = np.genfromtxt(\"data/X_train.txt\",delimiter = None)\n",
    "Y = np.genfromtxt(\"data/Y_train.txt\",delimiter = None)\n",
    "Xte = np.genfromtxt(\"data/X_test.txt\",delimiter = None)\n",
    "rescale = ml.transforms.rescale\n",
    "Xt,Xv,Yt,Yv = ml.splitData(X,Y,0.80)\n",
    "Xt, args=ml.transforms.rescale(Xt)\n",
    "Xte,_=ml.transforms.rescale(Xte,args)\n",
    "Xv,_=ml.transforms.rescale(Xv,args)\n",
    "M = Xt.shape[0]\n",
    "Mv= Xv.shape[0]\n",
    "rforest = [None]*25\n",
    "YtHat = np.zeros((M,25))\n",
    "YvHat = np.zeros((Mv,25))\n",
    "\n",
    "class randomForest2(ml.base.classifier):\n",
    "    def __init__(self, learners):\n",
    "        self.learners=learners\n",
    "        self.classes=learners[0].classes\n",
    "    def predictSoft(self,X):\n",
    "        ysoft = np.zeros((X.shape[0],len(self.classes)));\n",
    "        for i in range(len(self.learners)): ysoft+=self.learners[i].predictSoft(X);\n",
    "        return ysoft/len(self.learners);\n",
    "\n",
    "for l in range(25):\n",
    "    Xi,Yi = ml.bootstrapData(Xt,Yt, M) # draw this member's random sample of data\n",
    "    rforest[l] = dt.treeClassify() # and train the model on that draw\n",
    "    rforest[l].train(Xi,Yi,maxDepth=25,nFeatures=3)\n",
    "    YtHat[:,l] = rforest[l].predict(Xt) # predict on training data\n",
    "    YvHat[:,l] = rforest[l].predict(Xv) # and validation data & save results\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "rf2 = randomForest2(rforest);\n",
    "print \"AUC Train: \",rf2.auc(Xt,Yt),\" Valid: \",rf2.auc(Xv,Yv)\n",
    "\n",
    "\n",
    "Ypred = rf2.predictSoft(Xte)   # make \"soft\" predictions from your learner  (Mx2 numpy array)\n",
    "np.savetxt('Yhat.txt', np.vstack( (np.arange(len(Ypred)) , Ypred[:,1]) ).T, \n",
    "           '%d, %.2f',header='ID,Prob1',comments='',delimiter=',');\n",
    "#print(\"Finished Random Forest with nFeatures =\"+str(i))\n",
    "print(\"Finished Random Forest\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trained finished: [0]\n",
      "Trained finished: [6]\n",
      "Trained finished: [13]\n",
      "Trained finished: [0, 6, 13]\n",
      "Features finished: [0]\n",
      "Features finished: [6]\n",
      "Features finished: [13]\n",
      "Features finished: [0, 6, 13]\n",
      "Finished KNN\n"
     ]
    }
   ],
   "source": [
    "#KNN\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import mltools as ml\n",
    "import mltools.dtree as dt;\n",
    "import sys\n",
    "sys.path.append('.')\n",
    "Xtr = X[:10000]\n",
    "Ytr = Y[:10000]\n",
    "\n",
    "Xva = X[10000:20000]\n",
    "Yva = Y[10000:20000]\n",
    "\n",
    "fs = [\n",
    "[0, 6, 13]\n",
    "]\n",
    "\n",
    "class Ensemble:\n",
    "    def __init__(self, learners, setfeatures = None):\n",
    "        self.ensemble = learners\n",
    "        self.features = setfeatures\n",
    "        self.classes = learners[0].classes\n",
    "\n",
    "    def predictSoft(self, X):\n",
    "        Ypreds = []\n",
    "        for i, l in enumerate(self.ensemble):\n",
    "            Xf = X[:,self.features[i]]\n",
    "            Ypreds.append(l.predictSoft(Xf))\n",
    "            print(\"Features finished: \" + str(self.features[i]))\n",
    "        return np.mean(Ypreds, axis = 0)\n",
    "\n",
    "\n",
    "ensemble = []\n",
    "\n",
    "for f in fs:\n",
    "    XtrF = X[:,f]\n",
    "    l = ml.knn.knnClassify(X=XtrF, Y=Y, K=8)\n",
    "    ensemble.append(l)\n",
    "\n",
    "    print(\"Trained finished: \" + str(f))\n",
    "\n",
    "e = Ensemble(ensemble, fs)\n",
    "\n",
    "Ypred = e.predictSoft(Xte)\n",
    "np.savetxt('Yhat.txt', np.vstack( (np.arange(len(Ypred)), Ypred[:,1])).T, '%d, %.2f', header='ID,Prob1', comments='')\n",
    "print(\"Finished KNN\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Train:  0.6459808872  Valid:  0.648453134619\n",
      "Finished Clustering\n"
     ]
    }
   ],
   "source": [
    "#Clustering\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import mltools as ml\n",
    "import matplotlib.pyplot as plt   # use matplotlib for plotting with inline plots\n",
    "%matplotlib inline\n",
    "\n",
    "X = np.genfromtxt(\"data/X_train.txt\",delimiter=' ')[:,0:2]\n",
    "#X = X[:,0:2]\n",
    "Y = np.genfromtxt(\"data/Y_train.txt\",delimiter=' ')\n",
    "Xt,Xv,Yt,Yv = ml.splitData(X,Y,0.80)\n",
    "\n",
    "Xe = np.genfromtxt('data/X_test.txt',delimiter=' ')[:,0:2]\n",
    "\n",
    "\n",
    "\n",
    "def auc(soft,Y):\n",
    "    \"\"\"Manual AUC function for applying to soft prediction vectors\"\"\"\n",
    "    indices = np.argsort(soft)         # sort data by score value\n",
    "    Y = Y[indices]\n",
    "    sorted_soft = soft[indices]\n",
    "    \n",
    "    # compute rank (averaged for ties) of sorted data\n",
    "    dif = np.hstack( ([True],np.diff(sorted_soft)!=0,[True]) )\n",
    "    r1  = np.argwhere(dif).flatten()\n",
    "    r2  = r1[0:-1] + 0.5*(r1[1:]-r1[0:-1]) + 0.5\n",
    "    rnk = r2[np.cumsum(dif[:-1])-1]\n",
    "    \n",
    "    # number of true negatives and positives\n",
    "    n0,n1 = sum(Y == 0), sum(Y == 1)\n",
    "    \n",
    "    # compute AUC using Mann-Whitney U statistic\n",
    "    result = (np.sum(rnk[Y == 1]) - n1 * (n1 + 1.0) / 2.0) / n1 / n0\n",
    "    return result\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "nClust = 16\n",
    "np.random.seed(0)\n",
    "Z,T,pZ,ll = ml.cluster.gmmEM(Xt[:10000,:],K=nClust,init='k++',max_iter=50)\n",
    "\n",
    "Cluster_GMM = ml.bayes.gaussClassify()\n",
    "# Manually copy the EM Gaussian components into the Gaussian Bayes Classifier:\n",
    "Cluster_GMM.classes = np.arange(nClust)\n",
    "Cluster_GMM.means = T['mu']\n",
    "Cluster_GMM.covars = [ T['sig'][:,:,i]+.05*np.eye(Xt.shape[1]) for i in range(nClust) ]\n",
    "Cluster_GMM.probs = T['pi']\n",
    "\n",
    "# Find cluster membership probabilities for each data set:\n",
    "XtC = Cluster_GMM.predictSoft(Xt)\n",
    "XvC = Cluster_GMM.predictSoft(Xv)\n",
    "XeC = Cluster_GMM.predictSoft(Xe)\n",
    "\n",
    "# Create extended feature set:  features X times membership probability for each cluster\n",
    "XtC2 = np.einsum('ij,ik->ijk',XtC,Xt).reshape((Xt.shape[0],Xt.shape[1]*nClust))\n",
    "XvC2 = np.einsum('ij,ik->ijk',XvC,Xv).reshape((Xv.shape[0],Xv.shape[1]*nClust))\n",
    "XeC2 = np.einsum('ij,ik->ijk',XeC,Xe).reshape((Xe.shape[0],Xe.shape[1]*nClust))\n",
    "\n",
    "# Regress (should really use a classifier...)\n",
    "linr2 = ml.linear.linearRegress(XtC2,Yt, reg=1e-3)\n",
    "Pt3 = linr2.predict(XtC2)[:,0]\n",
    "Pv3 = linr2.predict(XvC2)[:,0]\n",
    "Pe3 = linr2.predict(XeC2)[:,0]\n",
    "\n",
    "#toKaggle('Pe3.csv',Pe3)\n",
    "#print(Pe3)\n",
    "#print \"3: Clustered LinRegress: MSE ~\",linr2.mse(XvC2,Yv),'; AUC = ',auc(Pv3,Yv)\n",
    "print \"AUC Train: \",auc(Pt3,Yt),\" Valid: \",auc(Pv3,Yv)\n",
    "np.savetxt('Yhat.txt', np.vstack(( np.arange(len(Pe3)), Pe3[:])).T, '%d, %.2f', header='ID,Prob1', comments='')\n",
    "print(\"Finished Clustering\")"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
